/**
 * 
 */
package com.vanstone.imgapp;

import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.vanstone.common.ObjectDuplicateException;
import com.vanstone.common.component.id.IDFactory;
import com.vanstone.fs.api.FileException;
import com.vanstone.imgapp.api.img.Category;
import com.vanstone.imgapp.api.img.services.AlbumService;
import com.vanstone.imgapp.api.img.services.CategoryService;
import com.vanstone.imgapp.api.img.services.ImageFSBean;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
import edu.uci.ics.crawler4j.url.WebURL;

/**
 * @author shipeng <br />
 * 
 */
public class DataCollectMainApp {

  public static ClassPathXmlApplicationContext applicationContext;

  static {
    applicationContext = new ClassPathXmlApplicationContext(new String[] {"classpath*:/META-INF/vanstone/vanstone-**.xml", "classpath*:/core-context.xml", "classpath*:META-INF/vanstone-imgapp-**-comp.xml"});
    applicationContext.start();
  }

  /**
   * @param args
   */
  public static void main(String[] args) {

    String crawlStorageFolder = "/data/crawl/root" + IDFactory.generateID();
    CrawlConfig config = new CrawlConfig();
    config.setCrawlStorageFolder(crawlStorageFolder);

    PageFetcher pageFetcher = new PageFetcher(config);
    RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
    RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
    CrawlController controller;
    try {
      controller = new CrawlController(config, pageFetcher, robotstxtServer);
      controller.addSeed("http://www.xiaotuku.com/");
      controller.startNonBlocking(XiaotukuCrawler.class, 5);
    } catch (Exception e) {
      e.printStackTrace();
    }
  }

  public static class XiaotukuCrawler extends WebCrawler {
    public static final Pattern STATIC_FILTER_PATTERN = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g" + "|png|tiff?|mid|mp2|mp3|mp4" + "|wav|avi|mov|mpeg|ram|m4v|pdf" + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

    public static final Pattern WEB_RESOURCE_PATTERN = Pattern.compile("http://www.xiaotuku.com/.*\\.html");

    @Override
    public boolean shouldVisit(WebURL url) {
      Matcher matcher = STATIC_FILTER_PATTERN.matcher(url.getURL());
      if (matcher.matches()) {
        return false;
      }
      matcher = WEB_RESOURCE_PATTERN.matcher(url.getURL());
      if (matcher.matches()) {
        return true;
      }
      return false;
    }

    @Override
    public void visit(Page page) {
      System.out.println(page.getWebURL().getURL());
      try {
        Document document = Jsoup.parse(new String(page.getContentData(), page.getContentCharset()));
        Element titleElement = document.getElementById("txtTitle");
        if (titleElement == null || titleElement.text() == null || titleElement.text().equals("")) {
          return;
        }
        String title = titleElement.text();
        String brief = title;
        String content = title;
        List<String> tagnames = new ArrayList<>();
        List<ImageFSBean> webURLs = new ArrayList<>();
        Elements mod_tags_elements = document.getElementsByClass("mod_tags");
        if (mod_tags_elements != null && mod_tags_elements.size() > 0) {
          Element tag_element = mod_tags_elements.get(0);
          Elements tag_elements = tag_element.getElementsByTag("a");
          if (tag_elements != null && tag_elements.size() > 0) {
            for (Element element : tag_elements) {
              String tagname = element.text();
              if (tagname != null && !tagname.equals("")) {
                tagnames.add(tagname);
              }
            }
          }
        }
        Element eData_element = document.getElementById("eData");
        Elements dl_elements = eData_element.getElementsByTag("dl");
        if (dl_elements == null || dl_elements.size() <= 0) {
          return;
        }
        for (Element dl_ele : dl_elements) {
          Elements dd_elements = dl_ele.getElementsByTag("dd");
          if (dd_elements == null || dd_elements.size() <= 0) {
            return;
          }
          for (Element dd_element : dd_elements) {
            String url = dd_element.text();
            if (url == null || url.equals("")) {
              continue;
            }
            if (url.startsWith("http://")) {
              ImageFSBean bean = new ImageFSBean(url);
              bean.setContent(content);
              webURLs.add(bean);
              break;
            }
          }
        }

        String categoryName = null;
        Elements art_info_elements = document.getElementsByClass("art_info");
        if (art_info_elements == null || art_info_elements.size() <= 0) {
          return;
        }
        Element art_info_element = art_info_elements.get(0);
        categoryName = art_info_element.getElementsByTag("a").get(0).text();
        if (categoryName == null || categoryName.equals("")) {
          return;
        }
        CategoryService categoryService = applicationContext.getBean(CategoryService.class);
        AlbumService albumService = applicationContext.getBean(AlbumService.class);
        Category category = categoryService.getCategoryByName(categoryName);
        if (category == null) {
          category = categoryService.addCategory(categoryName, null, null, null);
        }
        albumService.addAlbum(category.getId(), title, brief, content, tagnames, webURLs, true);
      } catch (UnsupportedEncodingException | ObjectDuplicateException | FileException e) {
        e.printStackTrace();
      }
    }
  }
}
